<!DOCTYPE html>
<html lang="en-US">
  <head>
    <meta charset="utf-8">
    <meta name="viewport" content="width=device-width,initial-scale=1">
    <title>kafka 为什么快 | Oliver知识收集站</title>
    <meta name="generator" content="VuePress 1.9.7">
    
    <meta name="description" content="享受着互联网广泛知识，并加以记录，日积月累让它成为一个档案处！">
    <meta name="viewport" content="width=device-width,initial-scale=1,user-scalable=no">
    
    <link rel="preload" href="/oliver-vuepress/assets/css/0.styles.4ea20d86.css" as="style"><link rel="preload" href="/oliver-vuepress/assets/js/app.c21e6ffc.js" as="script"><link rel="preload" href="/oliver-vuepress/assets/js/3.6dd9a2a1.js" as="script"><link rel="preload" href="/oliver-vuepress/assets/js/1.898920d0.js" as="script"><link rel="preload" href="/oliver-vuepress/assets/js/45.f692ec2d.js" as="script"><link rel="prefetch" href="/oliver-vuepress/assets/js/10.41b2bf91.js"><link rel="prefetch" href="/oliver-vuepress/assets/js/11.a95c117d.js"><link rel="prefetch" href="/oliver-vuepress/assets/js/12.8607f0e1.js"><link rel="prefetch" href="/oliver-vuepress/assets/js/13.a52d6846.js"><link rel="prefetch" href="/oliver-vuepress/assets/js/14.249b4e52.js"><link rel="prefetch" href="/oliver-vuepress/assets/js/15.d458d12e.js"><link rel="prefetch" href="/oliver-vuepress/assets/js/16.ba334206.js"><link rel="prefetch" href="/oliver-vuepress/assets/js/17.1b91c9fa.js"><link rel="prefetch" href="/oliver-vuepress/assets/js/18.e2ea2eb5.js"><link rel="prefetch" href="/oliver-vuepress/assets/js/19.bf0e2553.js"><link rel="prefetch" href="/oliver-vuepress/assets/js/20.268bd174.js"><link rel="prefetch" href="/oliver-vuepress/assets/js/21.cd1bbed5.js"><link rel="prefetch" href="/oliver-vuepress/assets/js/22.da4bc7f7.js"><link rel="prefetch" href="/oliver-vuepress/assets/js/23.12f0c72f.js"><link rel="prefetch" href="/oliver-vuepress/assets/js/24.b7886742.js"><link rel="prefetch" href="/oliver-vuepress/assets/js/25.6e71af85.js"><link rel="prefetch" href="/oliver-vuepress/assets/js/26.5c127243.js"><link rel="prefetch" href="/oliver-vuepress/assets/js/27.e98fd8bf.js"><link rel="prefetch" href="/oliver-vuepress/assets/js/28.ce83b09c.js"><link rel="prefetch" href="/oliver-vuepress/assets/js/29.50398f0f.js"><link rel="prefetch" href="/oliver-vuepress/assets/js/30.05e1339c.js"><link rel="prefetch" href="/oliver-vuepress/assets/js/31.ef4b13fb.js"><link rel="prefetch" href="/oliver-vuepress/assets/js/32.ba5f8351.js"><link rel="prefetch" href="/oliver-vuepress/assets/js/33.3902db0a.js"><link rel="prefetch" href="/oliver-vuepress/assets/js/34.36a05884.js"><link rel="prefetch" href="/oliver-vuepress/assets/js/35.87215872.js"><link rel="prefetch" href="/oliver-vuepress/assets/js/36.db360c58.js"><link rel="prefetch" href="/oliver-vuepress/assets/js/37.402e5374.js"><link rel="prefetch" href="/oliver-vuepress/assets/js/38.c9228dd8.js"><link rel="prefetch" href="/oliver-vuepress/assets/js/39.72ba5d1f.js"><link rel="prefetch" href="/oliver-vuepress/assets/js/4.7bb03d47.js"><link rel="prefetch" href="/oliver-vuepress/assets/js/40.7e7949bf.js"><link rel="prefetch" href="/oliver-vuepress/assets/js/41.c0d5b947.js"><link rel="prefetch" href="/oliver-vuepress/assets/js/42.d9984467.js"><link rel="prefetch" href="/oliver-vuepress/assets/js/43.e6a43668.js"><link rel="prefetch" href="/oliver-vuepress/assets/js/44.10d7fe47.js"><link rel="prefetch" href="/oliver-vuepress/assets/js/46.9b920343.js"><link rel="prefetch" href="/oliver-vuepress/assets/js/47.8e3d94f9.js"><link rel="prefetch" href="/oliver-vuepress/assets/js/48.7d356e5b.js"><link rel="prefetch" href="/oliver-vuepress/assets/js/49.b0df6271.js"><link rel="prefetch" href="/oliver-vuepress/assets/js/5.1fa544da.js"><link rel="prefetch" href="/oliver-vuepress/assets/js/50.805e1466.js"><link rel="prefetch" href="/oliver-vuepress/assets/js/51.1b31d40e.js"><link rel="prefetch" href="/oliver-vuepress/assets/js/52.44e69a41.js"><link rel="prefetch" href="/oliver-vuepress/assets/js/53.da1def53.js"><link rel="prefetch" href="/oliver-vuepress/assets/js/54.6569f7db.js"><link rel="prefetch" href="/oliver-vuepress/assets/js/55.5fc3de47.js"><link rel="prefetch" href="/oliver-vuepress/assets/js/56.da649377.js"><link rel="prefetch" href="/oliver-vuepress/assets/js/57.6ff15ed4.js"><link rel="prefetch" href="/oliver-vuepress/assets/js/58.a62f6424.js"><link rel="prefetch" href="/oliver-vuepress/assets/js/59.f68ae517.js"><link rel="prefetch" href="/oliver-vuepress/assets/js/6.f5bd8e9b.js"><link rel="prefetch" href="/oliver-vuepress/assets/js/60.dda416bc.js"><link rel="prefetch" href="/oliver-vuepress/assets/js/61.4e0c719f.js"><link rel="prefetch" href="/oliver-vuepress/assets/js/62.8c5ef01e.js"><link rel="prefetch" href="/oliver-vuepress/assets/js/63.7089eb8b.js"><link rel="prefetch" href="/oliver-vuepress/assets/js/64.b5ec150d.js"><link rel="prefetch" href="/oliver-vuepress/assets/js/65.6720cda4.js"><link rel="prefetch" href="/oliver-vuepress/assets/js/66.4ee90e29.js"><link rel="prefetch" href="/oliver-vuepress/assets/js/67.cc4b0c6d.js"><link rel="prefetch" href="/oliver-vuepress/assets/js/7.d5950c53.js"><link rel="prefetch" href="/oliver-vuepress/assets/js/8.382fb3a5.js"><link rel="prefetch" href="/oliver-vuepress/assets/js/9.d593f4c1.js">
    <link rel="stylesheet" href="/oliver-vuepress/assets/css/0.styles.4ea20d86.css">
  </head>
  <body>
    <div id="app" data-server-rendered="true"><div class="theme-container" data-v-130b300a><div data-v-130b300a><div class="password-shadow password-wrapper-out" style="display:none;" data-v-25ba6db2 data-v-130b300a data-v-130b300a><h3 class="title" data-v-25ba6db2 data-v-25ba6db2>Oliver知识收集站</h3> <p class="description" data-v-25ba6db2 data-v-25ba6db2>享受着互联网广泛知识，并加以记录，日积月累让它成为一个档案处！</p> <label id="box" class="inputBox" data-v-25ba6db2 data-v-25ba6db2><input type="password" value="" data-v-25ba6db2> <span data-v-25ba6db2>Konck! Knock!</span> <button data-v-25ba6db2>OK</button></label> <div class="footer" data-v-25ba6db2 data-v-25ba6db2><span data-v-25ba6db2><i class="iconfont reco-theme" data-v-25ba6db2></i> <a target="blank" href="https://vuepress-theme-reco.recoluan.com" data-v-25ba6db2>vuePress-theme-reco</a></span> <span data-v-25ba6db2><i class="iconfont reco-copyright" data-v-25ba6db2></i> <a data-v-25ba6db2><span data-v-25ba6db2>oliver.shi</span>
            
          <!---->
          2022
        </a></span></div></div> <div class="hide" data-v-130b300a><header class="navbar" data-v-130b300a><div class="sidebar-button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" role="img" viewBox="0 0 448 512" class="icon"><path fill="currentColor" d="M436 124H12c-6.627 0-12-5.373-12-12V80c0-6.627 5.373-12 12-12h424c6.627 0 12 5.373 12 12v32c0 6.627-5.373 12-12 12zm0 160H12c-6.627 0-12-5.373-12-12v-32c0-6.627 5.373-12 12-12h424c6.627 0 12 5.373 12 12v32c0 6.627-5.373 12-12 12zm0 160H12c-6.627 0-12-5.373-12-12v-32c0-6.627 5.373-12 12-12h424c6.627 0 12 5.373 12 12v32c0 6.627-5.373 12-12 12z"></path></svg></div> <a href="/oliver-vuepress/" class="home-link router-link-active"><!----> <span class="site-name">Oliver知识收集站</span></a> <div class="links"><div class="color-picker"><a class="color-button"><i class="iconfont reco-color"></i></a> <div class="color-picker-menu" style="display:none;"><div class="mode-options"><h4 class="title">Choose mode</h4> <ul class="color-mode-options"><li class="dark">dark</li><li class="auto active">auto</li><li class="light">light</li></ul></div></div></div> <div class="search-box"><i class="iconfont reco-search"></i> <input aria-label="Search" autocomplete="off" spellcheck="false" value=""> <!----></div> <nav class="nav-links can-hide"><div class="nav-item"><a href="/oliver-vuepress/" class="nav-link"><i class="undefined"></i>
  主页
</a></div><div class="nav-item"><div class="dropdown-wrapper"><a class="dropdown-title"><span class="title"><i class="undefined"></i>
      Java
    </span> <span class="arrow right"></span></a> <ul class="nav-dropdown" style="display:none;"><li class="dropdown-item"><!----> <a href="/oliver-vuepress/articles/java/basics/" class="nav-link"><i class="undefined"></i>
  基础
</a></li><li class="dropdown-item"><!----> <a href="/oliver-vuepress/articles/java/concurrent/" class="nav-link"><i class="undefined"></i>
  并发
</a></li><li class="dropdown-item"><!----> <a href="/oliver-vuepress/articles/java/jvm/jvm.html" class="nav-link"><i class="undefined"></i>
  JVM
</a></li><li class="dropdown-item"><!----> <a href="/oliver-vuepress/articles/java/other/" class="nav-link"><i class="undefined"></i>
  杂
</a></li></ul></div></div><div class="nav-item"><a href="/oliver-vuepress/articles/spring/first.html" class="nav-link"><i class="undefined"></i>
  Spring
</a></div><div class="nav-item"><div class="dropdown-wrapper"><a class="dropdown-title"><span class="title"><i class="undefined"></i>
      中间件
    </span> <span class="arrow right"></span></a> <ul class="nav-dropdown" style="display:none;"><li class="dropdown-item"><!----> <a href="/oliver-vuepress/articles/middleware/redis/redis.html" class="nav-link"><i class="undefined"></i>
  Redis
</a></li><li class="dropdown-item"><!----> <a href="/oliver-vuepress/articles/middleware/kafka/framework.html" class="nav-link"><i class="undefined"></i>
  Kafka
</a></li><li class="dropdown-item"><!----> <a href="/oliver-vuepress/articles/middleware/zookeeper.html" class="nav-link"><i class="undefined"></i>
  Zookeeper
</a></li></ul></div></div><div class="nav-item"><a href="/oliver-vuepress/articles/algorithm/" class="nav-link"><i class="undefined"></i>
  算法
</a></div><div class="nav-item"><a href="/oliver-vuepress/timeline/" class="nav-link"><i class="iconfont reco-date"></i>
  TimeLine
</a></div><div class="nav-item"><div class="dropdown-wrapper"><a class="dropdown-title"><span class="title"><i class="undefined"></i>
      收集站
    </span> <span class="arrow right"></span></a> <ul class="nav-dropdown" style="display:none;"><li class="dropdown-item"><!----> <a href="/oliver-vuepress/articles/collect/article/first.html" class="nav-link"><i class="undefined"></i>
  技术好文
</a></li><li class="dropdown-item"><!----> <a href="/oliver-vuepress/articles/collect/book/first.html" class="nav-link"><i class="undefined"></i>
  书籍
</a></li><li class="dropdown-item"><!----> <a href="/oliver-vuepress/articles/collect/assembly/first.html" class="nav-link"><i class="undefined"></i>
  优秀开发组件
</a></li><li class="dropdown-item"><!----> <a href="/oliver-vuepress/articles/collect/software/first.html" class="nav-link"><i class="undefined"></i>
  软件
</a></li><li class="dropdown-item"><!----> <a href="/oliver-vuepress/articles/collect/plugin/first.html" class="nav-link"><i class="undefined"></i>
  插件
</a></li></ul></div></div> <!----></nav></div></header> <div class="sidebar-mask" data-v-130b300a></div> <aside class="sidebar" data-v-130b300a><div class="personal-info-wrapper" data-v-39576ba9 data-v-130b300a><!----> <h3 class="name" data-v-39576ba9>
    oliver.shi
  </h3> <div class="num" data-v-39576ba9><div data-v-39576ba9><h3 data-v-39576ba9>52</h3> <h6 data-v-39576ba9>Articles</h6></div> <div data-v-39576ba9><h3 data-v-39576ba9>6</h3> <h6 data-v-39576ba9>Tags</h6></div></div> <ul class="social-links" data-v-39576ba9></ul> <hr data-v-39576ba9></div> <nav class="nav-links"><div class="nav-item"><a href="/oliver-vuepress/" class="nav-link"><i class="undefined"></i>
  主页
</a></div><div class="nav-item"><div class="dropdown-wrapper"><a class="dropdown-title"><span class="title"><i class="undefined"></i>
      Java
    </span> <span class="arrow right"></span></a> <ul class="nav-dropdown" style="display:none;"><li class="dropdown-item"><!----> <a href="/oliver-vuepress/articles/java/basics/" class="nav-link"><i class="undefined"></i>
  基础
</a></li><li class="dropdown-item"><!----> <a href="/oliver-vuepress/articles/java/concurrent/" class="nav-link"><i class="undefined"></i>
  并发
</a></li><li class="dropdown-item"><!----> <a href="/oliver-vuepress/articles/java/jvm/jvm.html" class="nav-link"><i class="undefined"></i>
  JVM
</a></li><li class="dropdown-item"><!----> <a href="/oliver-vuepress/articles/java/other/" class="nav-link"><i class="undefined"></i>
  杂
</a></li></ul></div></div><div class="nav-item"><a href="/oliver-vuepress/articles/spring/first.html" class="nav-link"><i class="undefined"></i>
  Spring
</a></div><div class="nav-item"><div class="dropdown-wrapper"><a class="dropdown-title"><span class="title"><i class="undefined"></i>
      中间件
    </span> <span class="arrow right"></span></a> <ul class="nav-dropdown" style="display:none;"><li class="dropdown-item"><!----> <a href="/oliver-vuepress/articles/middleware/redis/redis.html" class="nav-link"><i class="undefined"></i>
  Redis
</a></li><li class="dropdown-item"><!----> <a href="/oliver-vuepress/articles/middleware/kafka/framework.html" class="nav-link"><i class="undefined"></i>
  Kafka
</a></li><li class="dropdown-item"><!----> <a href="/oliver-vuepress/articles/middleware/zookeeper.html" class="nav-link"><i class="undefined"></i>
  Zookeeper
</a></li></ul></div></div><div class="nav-item"><a href="/oliver-vuepress/articles/algorithm/" class="nav-link"><i class="undefined"></i>
  算法
</a></div><div class="nav-item"><a href="/oliver-vuepress/timeline/" class="nav-link"><i class="iconfont reco-date"></i>
  TimeLine
</a></div><div class="nav-item"><div class="dropdown-wrapper"><a class="dropdown-title"><span class="title"><i class="undefined"></i>
      收集站
    </span> <span class="arrow right"></span></a> <ul class="nav-dropdown" style="display:none;"><li class="dropdown-item"><!----> <a href="/oliver-vuepress/articles/collect/article/first.html" class="nav-link"><i class="undefined"></i>
  技术好文
</a></li><li class="dropdown-item"><!----> <a href="/oliver-vuepress/articles/collect/book/first.html" class="nav-link"><i class="undefined"></i>
  书籍
</a></li><li class="dropdown-item"><!----> <a href="/oliver-vuepress/articles/collect/assembly/first.html" class="nav-link"><i class="undefined"></i>
  优秀开发组件
</a></li><li class="dropdown-item"><!----> <a href="/oliver-vuepress/articles/collect/software/first.html" class="nav-link"><i class="undefined"></i>
  软件
</a></li><li class="dropdown-item"><!----> <a href="/oliver-vuepress/articles/collect/plugin/first.html" class="nav-link"><i class="undefined"></i>
  插件
</a></li></ul></div></div> <!----></nav> <ul class="sidebar-links"><li><section class="sidebar-group collapsable depth-0"><p class="sidebar-heading open"><span>Kafka</span> <span class="arrow down"></span></p> <ul class="sidebar-links sidebar-group-items"><li><a href="/oliver-vuepress/articles/middleware/kafka/framework.html" class="sidebar-link">kafka 架构体系</a></li><li><a href="/oliver-vuepress/articles/middleware/kafka/producer.html" class="sidebar-link">kafka 生产者</a></li><li><a href="/oliver-vuepress/articles/middleware/kafka/consumer.html" class="sidebar-link">kafka 消费者</a></li><li><a href="/oliver-vuepress/articles/middleware/kafka/log.html" class="sidebar-link">kafka 日志</a></li><li><a href="/oliver-vuepress/articles/middleware/kafka/why-fast.html" aria-current="page" class="active sidebar-link">kafka 为什么快</a></li><li><a href="/oliver-vuepress/articles/middleware/kafka/application.html" class="sidebar-link">kafka的优缺点&amp; 应用场景</a></li></ul></section></li></ul> </aside> <div class="password-shadow password-wrapper-in" style="display:none;" data-v-25ba6db2 data-v-130b300a><h3 class="title" data-v-25ba6db2 data-v-25ba6db2>kafka 为什么快</h3> <!----> <label id="box" class="inputBox" data-v-25ba6db2 data-v-25ba6db2><input type="password" value="" data-v-25ba6db2> <span data-v-25ba6db2>Konck! Knock!</span> <button data-v-25ba6db2>OK</button></label> <div class="footer" data-v-25ba6db2 data-v-25ba6db2><span data-v-25ba6db2><i class="iconfont reco-theme" data-v-25ba6db2></i> <a target="blank" href="https://vuepress-theme-reco.recoluan.com" data-v-25ba6db2>vuePress-theme-reco</a></span> <span data-v-25ba6db2><i class="iconfont reco-copyright" data-v-25ba6db2></i> <a data-v-25ba6db2><span data-v-25ba6db2>oliver.shi</span>
            
          <!---->
          2022
        </a></span></div></div> <div data-v-130b300a><main class="page"><section><div class="page-title"><h1 class="title">kafka 为什么快</h1> <div data-v-f875f3fc><i class="iconfont reco-account" data-v-f875f3fc><span data-v-f875f3fc>oliver.shi</span></i> <i class="iconfont reco-date" data-v-f875f3fc><span data-v-f875f3fc>12/5/2021</span></i> <!----> <i class="tags iconfont reco-tag" data-v-f875f3fc><span class="tag-item" data-v-f875f3fc>消息队列</span><span class="tag-item" data-v-f875f3fc>kafka</span></i></div></div> <div class="theme-reco-content content__default"><h2 id="顺序写入"><a href="#顺序写入" class="header-anchor">#</a> 顺序写入</h2> <blockquote><p>操作系统可以针对线性读写做深层次的优化，比如预读(read-ahead，提前将一个比较大的磁盘块<br>
读入内存) 和后写(write-behind，将很多小的逻辑写操作合并起来组成一个大的物理写操作)技术。</p></blockquote> <p><strong>各个存储介质的速度层级,  理论上磁盘写入的速度相对来说慢的 但实际上 顺序写入数据的速度未必很慢</strong></p> <img src="/oliver-vuepress/middleware/kafka/fast1.jpg" alt="foo"> <p><strong>案例数据</strong>  一个由 7200r/min RAID阵列组成的磁盘簇的线性（顺序）写入速度可以达到 600MB/s，而随机写入速度只有 lOOKB/s，两性能相 6000 倍</p> <p><strong>Kafka 使用了是 日志文件尾部追加新消息，并且也不允许修改已写入的消息，这种方式属于典型的顺序写盘的操作</strong></p> <p><strong>一次写入操作</strong>
完成一次磁盘 IO，需要经过寻道、旋转和数据传输三个步骤。</p> <ul><li>寻道时间： 指将读写磁头移动至正确的磁道上所需要的时间</li> <li>旋转延迟： 指盘片旋转将请求数据所在的扇区移动到读写磁盘下方所需要的时间  （取决于 磁盘的转速）</li> <li>数据传输时间：指完成传输所请求的数据所需要的时间 （取决于 接口传输速率）</li></ul> <blockquote><p>顺序写入的操作：减少了 寻道、旋转的步骤</p></blockquote> <h2 id="pagecache-页缓存"><a href="#pagecache-页缓存" class="header-anchor">#</a> PageCache 页缓存</h2> <blockquote><p>页缓存时操作系统实现的一种主要的磁盘缓存，以此用来减少对磁盘I/O 的操作 。 具体来说，就是把磁盘中的数据缓存到内存中<br>
把对磁盘的访问变为对内存的访问。</p></blockquote> <p><strong>线程进行操作时</strong></p> <ul><li>读操作：
<ul><li>读取数据时，会直接在 页缓存（pagecache）中进行查找</li> <li>存在则直接返回数据，从而避免I/O 操作</li> <li>不存在，则向磁盘发起读取请求并将读取的数据页存入 页缓存</li></ul></li> <li>写操作
<ul><li>操作系统也会检测数据对应的页是否在页缓存中，如果不存在，则会先在页缓存中添加相应的页，最后将数据写入对应的页</li> <li>被修改过后的页也就变成了脏页，操作系统会在合适的时间把脏页中的数据写入磁盘，以保持数据的一致性。</li></ul></li></ul> <p><strong>页缓存 和 Mysql  InnoDB 的 内存架构类似</strong></p> <p><strong>总结: 消息先被写入页缓存，由操作系统负责刷盘任务。</strong></p> <p><strong>Linux 操作的脏页刷盘命令</strong></p> <ul><li><code>vm.dirtybackground.ratio</code> ：指定脏页数量到达系统内存的百分之多少触发 <code>pdflush/flush/kdmflush</code>等后台回写进程的运行来理脏页</li> <li><code>vm.dirtyratio</code> :  参数它用来指定当脏页数达到系统内存的百分之多少之后就不得不开始对脏页进行处理在此过程中，新的I/O请求会被阻挡直至所有脏页被冲刷到磁盘中</li></ul> <p><strong>Kafka为什么不自己管理缓存，而非要用page cache？原因有如下三点：</strong></p> <ul><li>JVM中一切皆对象，数据的对象存储会带来所谓object overhead，浪费空间；</li> <li>如果由JVM来管理缓存，会受到GC的影响，并且过大的堆也会拖累GC的效率，降低吞吐量；</li> <li>一旦程序崩溃，自己管理的缓存数据会全部丢失。</li></ul> <h2 id="零拷贝机制"><a href="#零拷贝机制" class="header-anchor">#</a> 零拷贝机制</h2> <blockquote><p>零拷贝 （Zero-copy）是指数据直接从磁盘文件复制到网卡设备中，而不需要经由应用程序之手。零拷贝大大提高了应用程序的性能，减少了内核和用户模式之间的上下文切换<br>
常见的将 文件发送个用户的流程  传统IO比如：读取文件，socket发送  需要经过4次 copy</p></blockquote> <p><strong>在不使用零拷贝的技术的流程下</strong> <img src="/oliver-vuepress/middleware/kafka/fast2.jpg" alt="foo"></p> <ul><li>步骤：
<ul><li>(1) 调用 read() 时，文件中的内容被复制到了内核模式下的Read Buffer 中。（操作系统内核缓冲区）</li> <li>(2) CPU 控制将内核模式数据复制到用户模式下</li> <li>(3）调用 write() 时，将用户模式下的容复制到内核模式下的 Socket Buffer 中</li> <li>(4）将内核模式下的 Socket Buffe 的数据复制到网卡设备中传送</li></ul></li></ul> <p><strong>使用零拷贝的技术的流程</strong> <img src="/oliver-vuepress/middleware/kafka/fast3.jpg" alt="foo"></p> <ul><li>步骤：
<ul><li>(1) 调用 read() 时，文件中的内容被复制到了内核模式下的Read Buffer 中。（操作系统内核缓冲区）</li> <li>(2) 将内核模式下的 Socket Buffe 的数据复制到网卡设备中传送</li></ul></li></ul> <p><strong>对于kafka来说  主要是优化了kafka的两个过程：</strong></p> <ol><li>网络数据持久化到磁盘 (Producer 到 Broker)</li> <li>磁盘文件通过网络发送（Broker 到 Consumer）</li></ol> <h3 id="网络数据持久化到磁盘-producer-到-broker"><a href="#网络数据持久化到磁盘-producer-到-broker" class="header-anchor">#</a> 网络数据持久化到磁盘 (Producer 到 Broker)</h3> <blockquote><p>对于kafka来说，Producer生产的数据存到broker，这个过程读取到socket buffer的网络数据，其实可以直接在OS内核缓冲区，完成落盘。并没有必要将socket buffer的网络数据，读取到应用进程缓冲区；在这里应用进程缓冲区其实就是broker，broker收到生产者的数据，就是为了持久化。</p></blockquote> <p><strong>在 Producer 到 Broker 写入数据中使用了 <code>mmap</code>  进行加快写入操作</strong>  (<code>mmap</code>将磁盘文件映射到内存，支持读和写，对内存的操作会反映在磁盘文件上。)</p> <h3 id="磁盘文件通过网络发送-broker-到-consumer"><a href="#磁盘文件通过网络发送-broker-到-consumer" class="header-anchor">#</a> 磁盘文件通过网络发送（Broker 到 Consumer）</h3> <blockquote><p>磁盘数据 通过 <code>DMA</code>  直接将存储器访问的 数据 拷贝到 内核态 buffer 中, <code>DMA</code> 拷贝到 NIC Buffer(socket buffer)，无需 CPU 拷贝整个流程上 减少了数据拷贝，整个读取文件 —》 网络发送 由一个 Sendfile 调用完成，整个过程只有两次上下文切换，因此大大提供了性能</p></blockquote> <ul><li>操作系统：底层依赖  <code>sendfile()</code>  方法实现</li> <li>Java语言：<code>FieChannal.transferTo()</code> 方法的底层实现就是 <code>sendfile()</code>  方法</li></ul> <p><strong>底层技术</strong></p> <blockquote><p>零拷贝技术通过DMA(Direct Memory Access）技术将文件内容复制到内核模式下的ReadBuffer。不过没有数据被复制到SockeBuffer</p></blockquote> <p><strong>总结</strong></p> <blockquote><p>零拷贝 不是真的零拷贝，而是适当的减少无用的拷贝，减少CPU上下文切换<br>
Kafka 的 Producer 使用 <code>MMAP</code>  技术实现的<br>
Kafka 的 Consumer 使用 <code>sendfile</code> 技术实现的<br> <code>MMAP</code>  和  <code>sendfile</code>  底层都是使用 <code>DMA</code>(Direct Memory Access，直接存储器）可以让外部设备不通过CPU而直接与系统内存交换数据的接口技术<br> <code>MMAP</code>  和   <code>sendfile</code>  是代码手段（逻辑方式）、   <code>DMA</code>  是 硬件手段 （物理方式）</p></blockquote> <p><strong>相关好文</strong></p> <ul><li><a href="https://www.cnblogs.com/rickiyang/p/13265043.html" target="_blank" rel="noopener noreferrer">零拷贝(Zero-copy) 浅析及其应用 - rickiyang - 博客园<span><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15" class="icon outbound"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path> <polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg> <span class="sr-only">(opens new window)</span></span></a></li></ul> <h2 id="批量与压缩"><a href="#批量与压缩" class="header-anchor">#</a> 批量与压缩</h2> <p><strong>批量</strong> <img src="/oliver-vuepress/middleware/kafka/fast4.jpg" alt="foo"></p> <blockquote><p>Kafka Producer 向 Broker 发送消息不是一条消息一条消息的发送。使用过 Kafka 的同学应该知道，Producer 有两个重要的参数：<code>batch.size</code> 和<code>linger.ms</code> 。这两个参数就和 Producer 的批量发送有关。</p></blockquote> <p><strong>压缩</strong>
Kafka 支持多种压缩算法： lz4、snappy、gzip、Kafka</p> <p><strong>Producer、Broker 和 Consumer 使用相同的压缩算法</strong>:
这样在 producer 向 Broker 写入数据，Consumer 向 Broker 读取数据时甚至可以不用解压缩， <strong>只要最终在 <code>Consumer</code>  端 Poll到消息时才解压</strong>, 这样节省了大量的网络和磁盘开销</p> <h2 id="分区并发"><a href="#分区并发" class="header-anchor">#</a> 分区并发</h2> <p>Kafka 的一个topic 可以分为 多个 parition,  只要在同一个 Consumer Group 中 的不同的 Consumer 并发的消费一个或多个 parition，每增加一个 Paritition 就增加了一个消费并发。 可以大大的提高并发的分区消费能里</p> <p><strong>并不是分区数越多越好</strong> <strong>分区数过度对导致什么问题？</strong></p> <ul><li><strong>越多的分区需要打开更多的文件句柄</strong>: 即分区越多，Broker 上需要打开的日志文件句柄越多</li> <li><strong>客服端服务端需要使用的内存就越多</strong>：分区越多，需要分区缓存的消息就越多</li> <li><strong>降低高可用性</strong>： 分区越多，每个Broker上分配的分区也就越多，当Broker 宕机，恢复时间将很长</li></ul> <h2 id="底层文件结构"><a href="#底层文件结构" class="header-anchor">#</a> 底层文件结构</h2> <p><a href="https://blog.csdn.net/gx11251143/article/details/107620259" target="_blank" rel="noopener noreferrer">Kafka对PageCache的使用_G-CSDN博客_kafka pagecache<span><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15" class="icon outbound"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path> <polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg> <span class="sr-only">(opens new window)</span></span></a></p></div></section> <footer class="page-edit"><!----> <div class="last-updated"><span class="prefix">上次更新: </span> <span class="time">4/21/2022, 8:17:59 AM</span></div></footer> <div class="page-nav"><p class="inner"><span class="prev"><a href="/oliver-vuepress/articles/middleware/kafka/log.html" class="prev">
            kafka 日志
          </a></span> <span class="next"><a href="/oliver-vuepress/articles/middleware/kafka/application.html">
            kafka的优缺点&amp; 应用场景
          </a></span></p></div> <div class="comments-wrapper"><!----></div> <ul class="side-bar sub-sidebar-wrapper" style="width:12rem;" data-v-cb1513f6><li class="level-2" data-v-cb1513f6><a href="/oliver-vuepress/articles/middleware/kafka/why-fast.html#顺序写入" class="sidebar-link reco-side-顺序写入" data-v-cb1513f6>顺序写入</a></li><li class="level-2" data-v-cb1513f6><a href="/oliver-vuepress/articles/middleware/kafka/why-fast.html#pagecache-页缓存" class="sidebar-link reco-side-pagecache-页缓存" data-v-cb1513f6>PageCache 页缓存</a></li><li class="level-2" data-v-cb1513f6><a href="/oliver-vuepress/articles/middleware/kafka/why-fast.html#零拷贝机制" class="sidebar-link reco-side-零拷贝机制" data-v-cb1513f6>零拷贝机制</a></li><li class="level-3" data-v-cb1513f6><a href="/oliver-vuepress/articles/middleware/kafka/why-fast.html#网络数据持久化到磁盘-producer-到-broker" class="sidebar-link reco-side-网络数据持久化到磁盘-producer-到-broker" data-v-cb1513f6>网络数据持久化到磁盘 (Producer 到 Broker)</a></li><li class="level-3" data-v-cb1513f6><a href="/oliver-vuepress/articles/middleware/kafka/why-fast.html#磁盘文件通过网络发送-broker-到-consumer" class="sidebar-link reco-side-磁盘文件通过网络发送-broker-到-consumer" data-v-cb1513f6>磁盘文件通过网络发送（Broker 到 Consumer）</a></li><li class="level-2" data-v-cb1513f6><a href="/oliver-vuepress/articles/middleware/kafka/why-fast.html#批量与压缩" class="sidebar-link reco-side-批量与压缩" data-v-cb1513f6>批量与压缩</a></li><li class="level-2" data-v-cb1513f6><a href="/oliver-vuepress/articles/middleware/kafka/why-fast.html#分区并发" class="sidebar-link reco-side-分区并发" data-v-cb1513f6>分区并发</a></li><li class="level-2" data-v-cb1513f6><a href="/oliver-vuepress/articles/middleware/kafka/why-fast.html#底层文件结构" class="sidebar-link reco-side-底层文件结构" data-v-cb1513f6>底层文件结构</a></li></ul></main> <!----></div></div></div></div><div class="global-ui"><div class="back-to-ceiling" style="right:1rem;bottom:6rem;width:2.5rem;height:2.5rem;border-radius:.25rem;line-height:2.5rem;display:none;" data-v-c6073ba8 data-v-c6073ba8><svg t="1574745035067" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="5404" class="icon" data-v-c6073ba8><path d="M526.60727968 10.90185116a27.675 27.675 0 0 0-29.21455937 0c-131.36607665 82.28402758-218.69155461 228.01873535-218.69155402 394.07834331a462.20625001 462.20625001 0 0 0 5.36959153 69.94390903c1.00431239 6.55289093-0.34802892 13.13561351-3.76865779 18.80351572-32.63518765 54.11355614-51.75690182 118.55860487-51.7569018 187.94566865a371.06718723 371.06718723 0 0 0 11.50484808 91.98906777c6.53300375 25.50556257 41.68394495 28.14064038 52.69160883 4.22606766 17.37162448-37.73630017 42.14135425-72.50938081 72.80769204-103.21549295 2.18761121 3.04276886 4.15646224 6.24463696 6.40373557 9.22774369a1871.4375 1871.4375 0 0 0 140.04691725 5.34970492 1866.36093723 1866.36093723 0 0 0 140.04691723-5.34970492c2.24727335-2.98310674 4.21612437-6.18497483 6.3937923-9.2178004 30.66633723 30.70611158 55.4360664 65.4791928 72.80769147 103.21549355 11.00766384 23.91457269 46.15860503 21.27949489 52.69160879-4.22606768a371.15156223 371.15156223 0 0 0 11.514792-91.99901164c0-69.36717486-19.13165746-133.82216804-51.75690182-187.92578088-3.42062944-5.66790279-4.76302748-12.26056868-3.76865837-18.80351632a462.20625001 462.20625001 0 0 0 5.36959269-69.943909c-0.00994388-166.08943902-87.32547796-311.81420293-218.6915546-394.09823051zM605.93803103 357.87693858a93.93749974 93.93749974 0 1 1-187.89594924 6.1e-7 93.93749974 93.93749974 0 0 1 187.89594924-6.1e-7z" p-id="5405" data-v-c6073ba8></path><path d="M429.50777625 765.63860547C429.50777625 803.39355007 466.44236686 1000.39046097 512.00932183 1000.39046097c45.56695499 0 82.4922232-197.00623328 82.5015456-234.7518555 0-37.75494459-36.9345906-68.35043303-82.4922232-68.34111062-45.57627738-0.00932239-82.52019037 30.59548842-82.51086798 68.34111062z" p-id="5406" data-v-c6073ba8></path></svg></div></div></div>
    <script src="/oliver-vuepress/assets/js/app.c21e6ffc.js" defer></script><script src="/oliver-vuepress/assets/js/3.6dd9a2a1.js" defer></script><script src="/oliver-vuepress/assets/js/1.898920d0.js" defer></script><script src="/oliver-vuepress/assets/js/45.f692ec2d.js" defer></script>
  </body>
</html>
